{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Evaluation.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "# *fairlib* Evaluation Tutorial\n"
      ],
      "metadata": {
        "id": "23VoXWy7LBr5"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 1. Installation"
      ],
      "metadata": {
        "id": "aiznwmbKLNbK"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wMe3UjWtK51q",
        "outputId": "acc4eff0-2742-4f83-9159-b7ae023b3057"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting fairlib\n",
            "  Downloading fairlib-0.0.3-py3-none-any.whl (63 kB)\n",
            "\u001b[K     |████████████████████████████████| 63 kB 2.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from fairlib) (1.21.6)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from fairlib) (1.0.2)\n",
            "Requirement already satisfied: seaborn in /usr/local/lib/python3.7/dist-packages (from fairlib) (0.11.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from fairlib) (4.64.0)\n",
            "Requirement already satisfied: docopt in /usr/local/lib/python3.7/dist-packages (from fairlib) (0.6.2)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from fairlib) (1.3.5)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from fairlib) (3.13)\n",
            "Collecting pickle5\n",
            "  Downloading pickle5-0.0.12-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (256 kB)\n",
            "\u001b[K     |████████████████████████████████| 256 kB 29.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from fairlib) (3.2.2)\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.18.0-py3-none-any.whl (4.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.0 MB 55.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from fairlib) (1.11.0+cu113)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->fairlib) (2.8.2)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->fairlib) (1.4.2)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->fairlib) (0.11.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->fairlib) (3.0.8)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib->fairlib) (4.2.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->matplotlib->fairlib) (1.15.0)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->fairlib) (2022.1)\n",
            "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->fairlib) (1.4.1)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->fairlib) (3.1.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->fairlib) (1.1.0)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.53.tar.gz (880 kB)\n",
            "\u001b[K     |████████████████████████████████| 880 kB 51.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers->fairlib) (2.23.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers->fairlib) (3.6.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers->fairlib) (4.11.3)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.5.1-py3-none-any.whl (77 kB)\n",
            "\u001b[K     |████████████████████████████████| 77 kB 6.9 MB/s \n",
            "\u001b[?25hCollecting PyYAML\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 60.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers->fairlib) (2019.12.20)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 54.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers->fairlib) (21.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers->fairlib) (3.8.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers->fairlib) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers->fairlib) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers->fairlib) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers->fairlib) (3.0.4)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers->fairlib) (7.1.2)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.53-py3-none-any.whl size=895260 sha256=c5c48dec8f73ec0d60ebce7e2289c19af0e22809669612f2359f236a871759ad\n",
            "  Stored in directory: /root/.cache/pip/wheels/87/39/dd/a83eeef36d0bf98e7a4d1933a4ad2d660295a40613079bafc9\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: PyYAML, tokenizers, sacremoses, huggingface-hub, transformers, pickle5, fairlib\n",
            "  Attempting uninstall: PyYAML\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed PyYAML-6.0 fairlib-0.0.3 huggingface-hub-0.5.1 pickle5-0.0.12 sacremoses-0.0.53 tokenizers-0.12.1 transformers-4.18.0\n"
          ]
        }
      ],
      "source": [
        "!pip install fairlib"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import fairlib"
      ],
      "metadata": {
        "id": "rdNyADX3LUbS"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir -p data/deepmoji\n",
        "!wget 'https://storage.googleapis.com/ai2i/nullspace/deepmoji/pos_pos.npy' -P 'data/deepmoji'\n",
        "!wget 'https://storage.googleapis.com/ai2i/nullspace/deepmoji/pos_neg.npy' -P 'data/deepmoji'\n",
        "!wget 'https://storage.googleapis.com/ai2i/nullspace/deepmoji/neg_pos.npy' -P 'data/deepmoji'\n",
        "!wget 'https://storage.googleapis.com/ai2i/nullspace/deepmoji/neg_neg.npy' -P 'data/deepmoji'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4KDwqmZELYCI",
        "outputId": "f97b0235-30ea-410c-e989-8341dc5dd485"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-05-07 15:30:11--  https://storage.googleapis.com/ai2i/nullspace/deepmoji/pos_pos.npy\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 108.177.127.128, 172.217.218.128, 142.251.18.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|108.177.127.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 405494864 (387M) [application/octet-stream]\n",
            "Saving to: ‘data/deepmoji/pos_pos.npy’\n",
            "\n",
            "pos_pos.npy         100%[===================>] 386.71M   139MB/s    in 2.8s    \n",
            "\n",
            "2022-05-07 15:30:14 (139 MB/s) - ‘data/deepmoji/pos_pos.npy’ saved [405494864/405494864]\n",
            "\n",
            "--2022-05-07 15:30:14--  https://storage.googleapis.com/ai2i/nullspace/deepmoji/pos_neg.npy\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 173.194.69.128, 108.177.127.128, 172.217.218.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|173.194.69.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 405504080 (387M) [application/octet-stream]\n",
            "Saving to: ‘data/deepmoji/pos_neg.npy’\n",
            "\n",
            "pos_neg.npy         100%[===================>] 386.72M   173MB/s    in 2.2s    \n",
            "\n",
            "2022-05-07 15:30:16 (173 MB/s) - ‘data/deepmoji/pos_neg.npy’ saved [405504080/405504080]\n",
            "\n",
            "--2022-05-07 15:30:16--  https://storage.googleapis.com/ai2i/nullspace/deepmoji/neg_pos.npy\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 142.251.18.128, 142.250.153.128, 74.125.128.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|142.251.18.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 405494864 (387M) [application/octet-stream]\n",
            "Saving to: ‘data/deepmoji/neg_pos.npy’\n",
            "\n",
            "neg_pos.npy         100%[===================>] 386.71M   136MB/s    in 2.9s    \n",
            "\n",
            "2022-05-07 15:30:19 (136 MB/s) - ‘data/deepmoji/neg_pos.npy’ saved [405494864/405494864]\n",
            "\n",
            "--2022-05-07 15:30:19--  https://storage.googleapis.com/ai2i/nullspace/deepmoji/neg_neg.npy\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 74.125.128.128, 173.194.69.128, 108.177.119.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|74.125.128.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 405504080 (387M) [application/octet-stream]\n",
            "Saving to: ‘data/deepmoji/neg_neg.npy’\n",
            "\n",
            "neg_neg.npy         100%[===================>] 386.72M   142MB/s    in 2.7s    \n",
            "\n",
            "2022-05-07 15:30:22 (142 MB/s) - ‘data/deepmoji/neg_neg.npy’ saved [405504080/405504080]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "fairlib.utils.seed_everything(2022)\n",
        "\n",
        "import numpy as np\n",
        "import os\n",
        "\n",
        "def read_data_file(input_file: str):\n",
        "    vecs = np.load(input_file)\n",
        "\n",
        "    np.random.shuffle(vecs)\n",
        "\n",
        "    return vecs[:40000], vecs[40000:42000], vecs[42000:44000]\n",
        "\n",
        "in_dir = \"data/deepmoji\"\n",
        "out_dir = \"data/deepmoji\"\n",
        "\n",
        "os.makedirs(out_dir, exist_ok=True)\n",
        "\n",
        "for split in ['pos_pos', 'pos_neg', 'neg_pos', 'neg_neg']:\n",
        "    train, dev, test = read_data_file(in_dir + '/' + split + '.npy')\n",
        "    for split_dir, data in zip(['train', 'dev', 'test'], [train, dev, test]):\n",
        "        os.makedirs(out_dir + '/' + split_dir, exist_ok=True)\n",
        "        np.save(out_dir + '/' + split_dir + '/' + split + '.npy', data)"
      ],
      "metadata": {
        "id": "sQYUwUJULX_k"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Train a Model"
      ],
      "metadata": {
        "id": "SOq2nWTeMUel"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "Shared_options = {\n",
        "    # The name of the dataset, corresponding dataloader will be used,\n",
        "    \"dataset\":  \"Moji\",\n",
        "\n",
        "    # Specifiy the path to the input data\n",
        "    \"data_dir\": \"data/deepmoji\",\n",
        "\n",
        "    # Device for computing, -1 is the cpu\n",
        "    \"device_id\":    -1,\n",
        "\n",
        "    # The default path for saving experimental results\n",
        "    \"results_dir\":  r\"results\",\n",
        "\n",
        "    # Will be used for saving experimental results\n",
        "    \"project_dir\":  r\"dev\",\n",
        "\n",
        "    # We will focusing on TPR GAP, implying the Equalized Odds for binary classification.\n",
        "    \"GAP_metric_name\":  \"TPR_GAP\",\n",
        "\n",
        "    # The overall performance will be measured as accuracy\n",
        "    \"Performance_metric_name\":  \"accuracy\",\n",
        "\n",
        "    # Model selections are based on DTO\n",
        "    \"selection_criterion\":  \"DTO\",\n",
        "\n",
        "    # Default dirs for saving checkpoints\n",
        "    \"checkpoint_dir\":   \"models\",\n",
        "    \"checkpoint_name\":  \"checkpoint_epoch\",\n",
        "\n",
        "    # Loading experimental results\n",
        "    \"n_jobs\":   1,\n",
        "}"
      ],
      "metadata": {
        "id": "5MDAksOZLX83"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "args = {\n",
        "    \"dataset\":Shared_options[\"dataset\"], \n",
        "    \"data_dir\":Shared_options[\"data_dir\"],\n",
        "    \"device_id\":Shared_options[\"device_id\"],\n",
        "\n",
        "    # Give a name to the exp, which will be used in the path\n",
        "    \"exp_id\":\"vanilla\",\n",
        "}\n",
        "\n",
        "# Init the argument\n",
        "options = fairlib.BaseOptions()\n",
        "state = options.get_state(args=args, silence=True)\n",
        "\n",
        "fairlib.utils.seed_everything(2022)\n",
        "\n",
        "# Init Model\n",
        "model = fairlib.networks.get_main_model(state)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EW4YIetDLX6Z",
        "outputId": "7cbe383a-135f-4c6e-d2ad-fffd278d2960"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:root:Unexpected args: ['-f', '/root/.local/share/jupyter/runtime/kernel-3dad1bfd-dd35-40d7-b985-35feaaff967a.json']\n",
            "INFO:root:Logging to ./results/dev/Moji/vanilla/output.log\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2022-05-07 15:30:29 [INFO ]  ======================================== 2022-05-07 15:30:29 ========================================\n",
            "2022-05-07 15:30:29 [INFO ]  Base directory is ./results/dev/Moji/vanilla\n",
            "Loaded data shapes: (99998, 2304), (99998,), (99998,)\n",
            "Loaded data shapes: (8000, 2304), (8000,), (8000,)\n",
            "Loaded data shapes: (7998, 2304), (7998,), (7998,)\n",
            "2022-05-07 15:30:30 [INFO ]  MLP( \n",
            "2022-05-07 15:30:30 [INFO ]    (output_layer): Linear(in_features=300, out_features=2, bias=True)\n",
            "2022-05-07 15:30:30 [INFO ]    (AF): Tanh()\n",
            "2022-05-07 15:30:30 [INFO ]    (hidden_layers): ModuleList(\n",
            "2022-05-07 15:30:30 [INFO ]      (0): Linear(in_features=2304, out_features=300, bias=True)\n",
            "2022-05-07 15:30:30 [INFO ]      (1): Tanh()\n",
            "2022-05-07 15:30:30 [INFO ]      (2): Linear(in_features=300, out_features=300, bias=True)\n",
            "2022-05-07 15:30:30 [INFO ]      (3): Tanh()\n",
            "2022-05-07 15:30:30 [INFO ]    )\n",
            "2022-05-07 15:30:30 [INFO ]    (criterion): CrossEntropyLoss()\n",
            "2022-05-07 15:30:30 [INFO ]  )\n",
            "2022-05-07 15:30:30 [INFO ]  Total number of parameters: 782402 \n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.train_self()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yTDzVz9mLX30",
        "outputId": "d4d7ebb5-81b3-4370-c7fd-2f45b3df3067"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2022-05-07 15:30:30 [INFO ]  Epoch:    0 [      0/  99998 ( 0%)]\tLoss: 0.6906\t Data Time: 0.02s\tTrain Time: 0.19s\n",
            "2022-05-07 15:30:34 [INFO ]  Epoch:    0 [  51200/  99998 (51%)]\tLoss: 0.3926\t Data Time: 0.36s\tTrain Time: 3.30s\n",
            "2022-05-07 15:30:38 [INFO ]  Evaluation at Epoch 0\n",
            "2022-05-07 15:30:38 [INFO ]  Validation accuracy: 72.55\tmacro_fscore: 72.44\tmicro_fscore: 72.55\tTPR_GAP: 40.07\tFPR_GAP: 40.07\tPPR_GAP: 39.10\t\n",
            "2022-05-07 15:30:38 [INFO ]  Test accuracy: 71.41\tmacro_fscore: 71.30\tmicro_fscore: 71.41\tTPR_GAP: 39.01\tFPR_GAP: 39.01\tPPR_GAP: 37.84\t\n",
            "2022-05-07 15:30:38 [INFO ]  Epoch:    1 [      0/  99998 ( 0%)]\tLoss: 0.4105\t Data Time: 0.01s\tTrain Time: 0.07s\n",
            "2022-05-07 15:30:42 [INFO ]  Epoch:    1 [  51200/  99998 (51%)]\tLoss: 0.4156\t Data Time: 0.37s\tTrain Time: 3.24s\n",
            "2022-05-07 15:30:46 [INFO ]  Evaluation at Epoch 1\n",
            "2022-05-07 15:30:46 [INFO ]  Validation accuracy: 72.36\tmacro_fscore: 72.32\tmicro_fscore: 72.36\tTPR_GAP: 39.81\tFPR_GAP: 39.81\tPPR_GAP: 39.27\t\n",
            "2022-05-07 15:30:46 [INFO ]  Test accuracy: 71.01\tmacro_fscore: 70.98\tmicro_fscore: 71.01\tTPR_GAP: 39.40\tFPR_GAP: 39.40\tPPR_GAP: 38.64\t\n",
            "2022-05-07 15:30:46 [INFO ]  Epoch:    2 [      0/  99998 ( 0%)]\tLoss: 0.3433\t Data Time: 0.01s\tTrain Time: 0.07s\n",
            "2022-05-07 15:30:49 [INFO ]  Epoch:    2 [  51200/  99998 (51%)]\tLoss: 0.3734\t Data Time: 0.38s\tTrain Time: 3.25s\n",
            "2022-05-07 15:30:53 [INFO ]  Epochs since last improvement: 1\n",
            "2022-05-07 15:30:53 [INFO ]  Evaluation at Epoch 2\n",
            "2022-05-07 15:30:53 [INFO ]  Validation accuracy: 72.42\tmacro_fscore: 72.37\tmicro_fscore: 72.42\tTPR_GAP: 40.91\tFPR_GAP: 40.91\tPPR_GAP: 40.20\t\n",
            "2022-05-07 15:30:53 [INFO ]  Test accuracy: 70.98\tmacro_fscore: 70.93\tmicro_fscore: 70.98\tTPR_GAP: 40.21\tFPR_GAP: 40.21\tPPR_GAP: 39.39\t\n",
            "2022-05-07 15:30:53 [INFO ]  Epoch:    3 [      0/  99998 ( 0%)]\tLoss: 0.3773\t Data Time: 0.01s\tTrain Time: 0.06s\n",
            "2022-05-07 15:30:57 [INFO ]  Epoch:    3 [  51200/  99998 (51%)]\tLoss: 0.3479\t Data Time: 0.38s\tTrain Time: 3.25s\n",
            "2022-05-07 15:31:01 [INFO ]  Epochs since last improvement: 2\n",
            "2022-05-07 15:31:01 [INFO ]  Evaluation at Epoch 3\n",
            "2022-05-07 15:31:01 [INFO ]  Validation accuracy: 72.09\tmacro_fscore: 71.92\tmicro_fscore: 72.09\tTPR_GAP: 41.54\tFPR_GAP: 41.54\tPPR_GAP: 40.17\t\n",
            "2022-05-07 15:31:01 [INFO ]  Test accuracy: 71.17\tmacro_fscore: 71.02\tmicro_fscore: 71.17\tTPR_GAP: 40.32\tFPR_GAP: 40.32\tPPR_GAP: 38.96\t\n",
            "2022-05-07 15:31:01 [INFO ]  Epoch:    4 [      0/  99998 ( 0%)]\tLoss: 0.3839\t Data Time: 0.02s\tTrain Time: 0.06s\n",
            "2022-05-07 15:31:05 [INFO ]  Epoch:    4 [  51200/  99998 (51%)]\tLoss: 0.3499\t Data Time: 0.38s\tTrain Time: 3.28s\n",
            "2022-05-07 15:31:11 [INFO ]  Epochs since last improvement: 3\n",
            "2022-05-07 15:31:11 [INFO ]  Evaluation at Epoch 4\n",
            "2022-05-07 15:31:11 [INFO ]  Validation accuracy: 71.50\tmacro_fscore: 71.43\tmicro_fscore: 71.50\tTPR_GAP: 42.76\tFPR_GAP: 42.76\tPPR_GAP: 42.00\t\n",
            "2022-05-07 15:31:11 [INFO ]  Test accuracy: 70.49\tmacro_fscore: 70.43\tmicro_fscore: 70.49\tTPR_GAP: 41.37\tFPR_GAP: 41.37\tPPR_GAP: 40.51\t\n",
            "2022-05-07 15:31:11 [INFO ]  Epoch:    5 [      0/  99998 ( 0%)]\tLoss: 0.3746\t Data Time: 0.03s\tTrain Time: 0.28s\n",
            "2022-05-07 15:31:15 [INFO ]  Epoch:    5 [  51200/  99998 (51%)]\tLoss: 0.3748\t Data Time: 0.37s\tTrain Time: 3.28s\n",
            "2022-05-07 15:31:19 [INFO ]  Epochs since last improvement: 4\n",
            "2022-05-07 15:31:19 [INFO ]  Evaluation at Epoch 5\n",
            "2022-05-07 15:31:19 [INFO ]  Validation accuracy: 72.67\tmacro_fscore: 72.60\tmicro_fscore: 72.67\tTPR_GAP: 39.17\tFPR_GAP: 39.17\tPPR_GAP: 38.35\t\n",
            "2022-05-07 15:31:19 [INFO ]  Test accuracy: 71.69\tmacro_fscore: 71.62\tmicro_fscore: 71.69\tTPR_GAP: 37.97\tFPR_GAP: 37.97\tPPR_GAP: 36.91\t\n",
            "2022-05-07 15:31:19 [INFO ]  Epoch:    6 [      0/  99998 ( 0%)]\tLoss: 0.3624\t Data Time: 0.01s\tTrain Time: 0.06s\n",
            "2022-05-07 15:31:23 [INFO ]  Epoch:    6 [  51200/  99998 (51%)]\tLoss: 0.3529\t Data Time: 0.38s\tTrain Time: 3.24s\n",
            "2022-05-07 15:31:26 [INFO ]  Epochs since last improvement: 5\n",
            "2022-05-07 15:31:27 [INFO ]  Evaluation at Epoch 6\n",
            "2022-05-07 15:31:27 [INFO ]  Validation accuracy: 72.70\tmacro_fscore: 72.62\tmicro_fscore: 72.70\tTPR_GAP: 38.29\tFPR_GAP: 38.29\tPPR_GAP: 37.50\t\n",
            "2022-05-07 15:31:27 [INFO ]  Test accuracy: 71.76\tmacro_fscore: 71.70\tmicro_fscore: 71.76\tTPR_GAP: 37.59\tFPR_GAP: 37.59\tPPR_GAP: 36.79\t\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "By default, fairlib print and save 6 metrics:\n",
        "- accuracy, macro F1 score, and micro F1 score, which are most commenly used evaluation metrics for performance evaluation.\n",
        "- rms aggregated TPR, FPR, and PPR GAP scores for fairness assesment. "
      ],
      "metadata": {
        "id": "t3tqL2F4Ori8"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Scenario 1: Confusion Matrix Based Metrics"
      ],
      "metadata": {
        "id": "YsTpEaDQPr3t"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "\n",
        "path = \"{results_dir}/{project_dir}/{dataset}/{exp_id}/{checkpoint_dir}/{checkpoint_name}{epoch}.pth.tar\"\n",
        "\n",
        "# Path to the first epoch\n",
        "path_vanilla_epoch0 = path.format(\n",
        "    exp_id = \"vanilla\",\n",
        "    epoch = \"0\",\n",
        "    results_dir=Shared_options[\"results_dir\"],\n",
        "    project_dir=Shared_options[\"project_dir\"],\n",
        "    dataset=Shared_options[\"dataset\"],\n",
        "    checkpoint_dir=Shared_options[\"checkpoint_dir\"],\n",
        "    checkpoint_name=Shared_options[\"checkpoint_name\"],\n",
        ")\n",
        "\n",
        "epoch_results = torch.load(path_vanilla_epoch0)\n",
        "# The keys for saved items\n",
        "print(epoch_results.keys())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ajx1OwLnLX1A",
        "outputId": "be0b990b-2579-4342-e848-0f7406598325"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "dict_keys(['epoch', 'epochs_since_improvement', 'loss', 'valid_confusion_matrices', 'test_confusion_matrices', 'dev_evaluations', 'test_evaluations'])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "*fairlib* saves confusion matrices for each protected groups as well as the overall confusion matrix. These matrices are stored in a dictionary, indexed with the group id."
      ],
      "metadata": {
        "id": "SDptnbzkVywW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "epoch_results[\"valid_confusion_matrices\"].keys()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HYfWfKqtLXuG",
        "outputId": "e3166d01-2cea-4ccd-bf00-35ce4fb015ff"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['overall', 0, 1])"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "epoch_results[\"valid_confusion_matrices\"][\"overall\"]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KDdN9v_SVrNE",
        "outputId": "de4a45bc-c2a5-4c19-ffed-f9c0f66e902c"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[2655, 1345],\n",
              "       [ 851, 3149]])"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from fairlib.src.evaluators.evaluator import confusion_matrix_based_scores"
      ],
      "metadata": {
        "id": "Ie6Z6X2LWsal"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "confusion_matrix_based_scores(epoch_results[\"valid_confusion_matrices\"][\"overall\"])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zN_WqRKAW2yN",
        "outputId": "738a930c-5f4c-4341-8971-ff336dbda489"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'ACC': array([0.7255, 0.7255]),\n",
              " 'FDR': array([0.24272676, 0.29928794]),\n",
              " 'FNR': array([0.33625, 0.21275]),\n",
              " 'FPR': array([0.21275, 0.33625]),\n",
              " 'NPV': array([0.70071206, 0.75727324]),\n",
              " 'PPR': array([0.43825, 0.56175]),\n",
              " 'PPV': array([0.75727324, 0.70071206]),\n",
              " 'TNR': array([0.78725, 0.66375]),\n",
              " 'TPR': array([0.66375, 0.78725])}"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from fairlib.src.evaluators.evaluator import power_mean"
      ],
      "metadata": {
        "id": "j55NEXFlXEcD"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "numbers = np.array([1,2,3,4,5])\n",
        "# generalized mean aggregation\n",
        "[\n",
        " power_mean(numbers, p=100), # Max\n",
        " power_mean(numbers, p=2), # Root Mean Square\n",
        " power_mean(numbers, p=1), # Arithmetic Mean\n",
        " power_mean(numbers, p=-100), # Min\n",
        "]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wNM5By0wXEZQ",
        "outputId": "0d4117e6-3c28-4b83-96ed-28cbbd6cc15a"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[5, 3.3166247903554, 3.0, 1]"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "- Max Violation\n",
        "- RMS GAP \n",
        "- Max Min Fairness"
      ],
      "metadata": {
        "id": "a6hUQfZDZOHN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from fairlib.src.evaluators.evaluator import Aggregation_GAP"
      ],
      "metadata": {
        "id": "vD_U3GP3ZNe5"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Confusion matrices of the vanilla model's first epoch over the vailidation dataset\n",
        "confusion_matrices = epoch_results[\"valid_confusion_matrices\"]\n",
        "\n",
        "# all_scores = dict()\n",
        "all_scores = {}\n",
        "\n",
        "# Overall evaluation\n",
        "all_scores[\"overall\"] = confusion_matrix_based_scores(confusion_matrices[\"overall\"])\n",
        "\n",
        "# Group scores\n",
        "distinct_groups = [0,1] # binary protected groups, AAE verse SAE\n",
        "for gid in distinct_groups:\n",
        "    group_confusion_matrix = confusion_matrices[gid]\n",
        "    all_scores[gid] = confusion_matrix_based_scores(group_confusion_matrix)"
      ],
      "metadata": {
        "id": "XA4jGay8bJlV"
      },
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Aggregation_GAP(\n",
        "    distinct_groups=[0,1], \n",
        "    all_scores=all_scores, \n",
        "\n",
        "    # Take the absolute different if None, \n",
        "    # using generalized mean aggregation if not None.\n",
        "    group_agg_power = -10,\n",
        "    \n",
        "    # RMS aggregation by default\n",
        "    class_agg_power=2, \n",
        "    metric=\"TPR\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hc5iGYhzXEWc",
        "outputId": "ee41622b-345b-4592-8954-1d2e0635853b"
      },
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.2003354978462722"
            ]
          },
          "metadata": {},
          "execution_count": 68
        }
      ]
    }
  ]
}